package edu.whu.rose.clock.kps.algorithm.blinks2.index;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;

import com.sleepycat.je.DatabaseException;

import edu.whu.rose.clock.kps.algorithm.banks1.SPVertex;
import edu.whu.rose.clock.kps.algorithm.blinks.index.Block;
import edu.whu.rose.clock.kps.algorithm.blinks.index.BlockKeyword;
import edu.whu.rose.clock.kps.algorithm.blinks.index.BlockVertexKeyword;
import edu.whu.rose.clock.kps.algorithm.blinks.index.GraphPartition;
import edu.whu.rose.clock.kps.algorithm.blinks.index.GroupIteratorInBlock;
import edu.whu.rose.clock.kps.algorithm.blinks.index.GroupSPVertex;
import edu.whu.rose.clock.kps.algorithm.cbes.DijkstraIteratorInSubgraph;
import edu.whu.rose.clock.kps.datastruct.SimpleSubgraphVertex;
import edu.whu.rose.clock.kps.graph.GraphMatrix;
import edu.whu.rose.clock.setup.IndexEnvironment;
import edu.whu.rose.clock.ssql.graphindex.GraphIndexManager;
import edu.whu.rose.clock.ssql.index.VertexIndexDB;
import edu.whu.rose.clock.ssql.util.SequentialIntArray;
import edu.whu.rose.clock.ssql.util.SubgraphMatrix;

public class BilevelIndexManager2 {

	private BilevelIndexDB2 bidb;

	private Block[] blocks;

	HashSet<String> words = new HashSet<String>();

	public BilevelIndexManager2(BilevelIndexDB2 bidb) {
		this.bidb = bidb;

		words.add("jeff");
		words.add("dynamic");
		words.add("optimal");
		words.add("sharmin");
		words.add("ahmed");
		words.add("trust");
		words.add("pervasive");
		words.add("analysis");
		words.add("pointers");
		words.add("structures");
		words.add("large");
		words.add("programs");
		
		words.add("dht-based");
		words.add("p2p");
		words.add("adaptive");
		words.add("scheduling");
		words.add("grid");
		words.add("jagadish");
		words.add("gene");
		words.add("protein");
		words.add("structure");
		words.add("adam");
		words.add("supervised");
		words.add("learning");
		words.add("database");
		words.add("richard");
		words.add("emily");
		words.add("song");
		words.add("charles");
		words.add("victor");
		words.add("web");
		words.add("congress");
		words.add("ieee");
		words.add("computer");
		words.add("society");
		words.add("oscar");
		
		words.add("multimedia");
		words.add("warehouse");
		words.add("image");
		words.add("video");
		words.add("content");
		
		words.add("integration");
		words.add("knowledge");
		words.add("philipp");
		words.add("paul");
		words.add("retrieval");
		words.add("james");
		words.add("edward");
		words.add("steven");
		words.add("elizabeth");
		words.add("scalable");
		words.add("recoverable");
		words.add("object");
		words.add("evolution");
		words.add("object-oriented");
		words.add("cad");
		words.add("algebra");
		words.add("information");
		words.add("visualization");
		words.add("eric");
		words.add("alien");
		words.add("3d");
		words.add("graphics");
		words.add("metadata");
		words.add("oois");
		words.add("schema");
		words.add("reuse");
		words.add("conference");
		words.add("peer-to-peer");
		words.add("self-organizing");
		
	}

	public static void main(String[] args) {
		File file = new File("F:/data/5000029_dbEnv_bilevel");
		if (file.isDirectory()) {
			File[] files = file.listFiles();
			for (int i = 0; i < files.length; i++) {
				files[i].delete();
			}
		}
		
		BilevelIndexDB2 bidb = new BilevelIndexDB2();
		if (bidb.initEnv() && bidb.openDB()) {
			BilevelIndexManager2 blim = new BilevelIndexManager2(bidb);
			blim.process(198467, "F:/data/5000029_graph/edge.txt", 300);

//			try {
//				FileWriter fw1 = new FileWriter("bilevel files/keyword_block.txt");
//				FileWriter fw2 = new FileWriter("bilevel files/portal_block.txt");
//				FileWriter fw3 = new FileWriter("bilevel files/keyword_node.txt");
//				FileWriter fw4 = new FileWriter("bilevel files/node_keyword.txt");
//				FileWriter fw5 = new FileWriter("bilevel files/portal_node.txt");
//				FileWriter fw6 = new FileWriter("bilevel files/node_portal.txt");
//				bidb.printKeywordBlock(fw1);
//				bidb.printPortalBlock(fw2);
//				bidb.printKeywordNode(fw3);
//				bidb.printNodeKeyword(fw4);
//				bidb.printPortalNode(fw5);
//				bidb.printNodePortal(fw6);
//				fw1.close();
//				fw2.close();
//				fw3.close();
//				fw4.close();
//				fw5.close();
//				fw6.close();
//			} catch (IOException ex) {
//				ex.printStackTrace();
//			}

			blim.close();
		}
	}
	
	public void close() {
		bidb.closeDB();
	}

	public boolean validate(int edgeNum, int vertexNum, HashMap<Integer, SequentialIntArray> portals) {
		int edgeSum = 0;
		int vertexSum = 0;
		for (int i = 0; i < blocks.length; i++) {
			edgeSum += blocks[i].edgeNum();
			vertexSum += blocks[i].size();
		}
		Iterator<SequentialIntArray> it = portals.values().iterator();
		while (it.hasNext()) {
			vertexSum -= it.next().getCapacity() - 1;
		}
		if (edgeSum == edgeNum && vertexSum == vertexNum) {
			return true;
		} else {
			return false;
		}
	}

	public void process(int order, String graphfile, int blocksize) {
		Date start = new Date();
		
		GraphMatrix graph = new GraphMatrix(order);
		graph.loadEdges(graphfile);

		GraphPartition gp = new GraphPartition(graph, blocksize);
		blocks = gp.process();
		int[] vertex_block = gp.getVertexBlock();
		int[][] edges = gp.findEdgeSeperator(blocks);
		HashMap<Integer, SequentialIntArray> portals = gp.producePortal(edges,
				blocks);
		if (validate(graph.getM(), graph.getN(), portals)) {
			System.out.println("pass");
		}
//		bidb.clearDB(1);
		buildKeywordBlockIndex("F:/data/5000029_graph/vertex.txt",
		vertex_block, portals);
		System.out.println("index 1 finished");
		
//		bidb.clearDB(2);
		buildPortalBlockIndex(portals);
		System.out.println("index 2 finished");
		
//		bidb.clearDB(3);
//		bidb.clearDB(4);
		buildKeywordVertexIndex(vertex_block, portals);
		System.out.println("index 3,4 finished");
		
//		bidb.clearDB(5);
		buildPortalVertexIndex();
		System.out.println("index 5 finished");
		
//		bidb.clearDB(6);
		buildVertexPortalIndex();
		System.out.println("index 6 finished");
		
		Date end = new Date();
		System.out.println("Time exhausted: " + (end.getTime() - start.getTime()) + "ms");
	}

	private void buildPortalBlockIndex(
			HashMap<Integer, SequentialIntArray> portals) {
		Iterator<Integer> it = portals.keySet().iterator();
		while (it.hasNext()) {
			Integer V = it.next();
			if (V.intValue() == 49760) {
				System.out.println();
			}
			SequentialIntArray blocklist = portals.get(V);
			try {
				bidb.insertToPortalBlock(V.intValue(), blocklist.getArray());
			} catch (DatabaseException ex) {
				ex.printStackTrace();
			}
		}
	}

	private void buildKeywordBlockIndex(String vertexFile, int[] vertex_block,
			HashMap<Integer, SequentialIntArray> portals) {
		KeywordBlockCache kbc = new KeywordBlockCache(bidb, 30000000);
		HashSet<String> stopwordlist = stopwordlist("stop word list.txt");

		int counter = 0;
		Date sDate = new Date();
		Date eDate = null;
		try {
			BufferedReader br = new BufferedReader(new FileReader(vertexFile));
			while (br.ready()) {
				String line = br.readLine().trim();
				if (line.length() > 0) {
					int pos = line.indexOf(" ");
					if (pos == -1) {
						System.out.println("Found wrong line: " + line);
						continue;
					}
					String key = line.substring(0, pos);
					String value = line.substring(pos).trim();

					// get corresponding block from vertex id (key)
					int vertex = new Integer(key).intValue();
					SequentialIntArray sia = portals.get(new Integer(vertex));
					if (sia == null) {
						int block = vertex_block[vertex - 1];
						this.insertKeywordBlock(block, vertex, value,
								stopwordlist, kbc);
					} else {
						for (int i = 0; i < sia.getCapacity(); i++) {
							int block = sia.get(i);
							this.insertKeywordBlock(block, vertex, value,
									stopwordlist, kbc);
						}
					}

					counter++;
					if (counter % 1000 == 0) {
						System.out.println(key);
						eDate = new Date();
						System.out.println("Time exhausted: "
								+ (eDate.getTime() - sDate.getTime()) + " ms");
						sDate = eDate;
						// kbc.statistic();
					}
				}
			}
			 kbc.flush();
			 kbc.statistic();
		} catch (UnsupportedEncodingException ex) {
			ex.printStackTrace();
		} catch (IOException ex) {
			ex.printStackTrace();
		}
		 catch (DatabaseException ex) {
			 ex.printStackTrace();
		 }
	}

	private void insertKeywordBlock(int block, int vertex, String value,
			HashSet<String> stopwordlist, KeywordBlockCache kbc) {
		String[] pairs = value.split("@@");
		for (int i = 0; i < pairs.length; i++) {
			String pair = pairs[i].trim();
			if (pair.length() > 0) {
				int pos = pair.indexOf(":");
				if (pos == -1) {
					System.out.println("Found wrong attr-value pair: " + pair);
					continue;
				}
				String attrName = pair.substring(0, pos).trim();
				String attrValue = pair.substring(pos + 1).trim();
				String[] keywords = null;
				if (attrName.equals("title")) {
					keywords = GraphIndexManager.parse(attrValue);
				} else if (attrName.equals("name")) {
					keywords = GraphIndexManager.naiveParse(attrValue);
				} else {
					continue;
				}
				for (int j = 0; j < keywords.length; j++) {
					String keyword = keywords[j].trim();
					String lcKeyword = keyword.toLowerCase();
					if (lcKeyword.length() > 0) {
						if (stopwordlist.contains(lcKeyword)) {
							continue;
						}
						 try {					
							 kbc.insert(lcKeyword, block);
						 }
						 catch (DatabaseException ex) {
							 ex.printStackTrace();
						 }
						 catch (UnsupportedEncodingException ex) {
							 ex.printStackTrace();
						 }
					}
				}
			}
		}
	}

	private void buildKeywordVertexIndex(int[] vertex_block, HashMap<Integer, SequentialIntArray> portals) {
		
		IndexEnvironment ie = new IndexEnvironment("dbEnv_vertexindex");
		ie.openEnv();
		VertexIndexDB vidb = new VertexIndexDB(ie.getEnvironment());
		if (!vidb.openDB()) {
			System.exit(1);
		}
		
		Iterator<String> it = words.iterator();
		while (it.hasNext()) {
			String keyword = it.next();
			
			int[] vertexes = vidb.search(keyword);
			HashMap<Integer, SequentialIntArray> data = new HashMap<Integer, SequentialIntArray>(8);
			for (int i = 0; i < vertexes.length; i++) {
				int vertex = vertexes[i];
				SequentialIntArray sia = portals.get(new Integer(vertex));
				if (sia == null) {
					int block = vertex_block[vertex - 1];
					SequentialIntArray index = data.get(new Integer(block));
					if (index != null) {
						index.insert(vertex);
					} 
					else {
						index = new SequentialIntArray(4);
						index.insert(vertex);
						data.put(new Integer(block), index);
					}
				} else {
					for (int j = 0; j < sia.getCapacity(); j++) {
						int block = sia.get(j);
						SequentialIntArray index = data.get(new Integer(block));
						if (index != null) {
							index.insert(vertex);
						} 
						else {
							index = new SequentialIntArray(4);
							index.insert(vertex);
							data.put(new Integer(block), index);
						}
					}
				}
			}
			
			// build index
			Iterator<Integer> it2 = data.keySet().iterator();
			while (it2.hasNext()) {
				Integer block = it2.next();
				SubgraphMatrix subgraph = new SubgraphMatrix(blocks[block.intValue() - 1]);
				SequentialIntArray sia = data.get(block);
				GroupIteratorInBlock giis = new GroupIteratorInBlock(subgraph, sia.getArray());
				ArrayList<KNEntry2> eia = new ArrayList<KNEntry2>();
				while (giis.hasNext()) {
					GroupSPVertex spv = giis.next();
					
					KNEntry2 kne = new KNEntry2((short)spv.getDistance(), spv.getID(), spv.getSource(), spv.getPredecessor());
					eia.add(kne);
					
					BlockVertexKeyword bvk = new BlockVertexKeyword(block.intValue(), spv.getID(), keyword);
					NKEntry2 nke = new NKEntry2((short)spv.getDistance(), spv.getSource(), spv.getPredecessor());
					try {
						bidb.insertToNodeKeyword(bvk, nke);
					}
					catch (DatabaseException ex) {
						ex.printStackTrace();
					}
				}
				BlockKeyword bk = new BlockKeyword(block.intValue(), keyword);
				try {
					bidb.insertToKeywordNode(bk, eia);
				} catch (DatabaseException ex) {
					ex.printStackTrace();
				}
			}
			
			System.out.println(keyword + " finished");
		}
		
		vidb.closeDB();
	}

//	private void insertKeywordVertex(int block, int vertex, String value) {
//		if (!words.contains(value)) {
//			return;
//		}
//		HashMap<Integer, SequentialIntArray> data = cache.get(value);
//		if (data != null) {
//			SequentialIntArray sia = data.get(new Integer(block));
//			if (sia != null) {
//				sia.insert(vertex);
//			} else {
//				sia = new SequentialIntArray(4);
//				sia.insert(vertex);
//				data.put(new Integer(block), sia);
//			}
//		} else {
//			data = new HashMap<Integer, SequentialIntArray>(8);
//			SequentialIntArray sia = new SequentialIntArray();
//			sia.insert(vertex);
//			data.put(new Integer(block), sia);
//			cache.put(value, data);
//		}
//	}

//	public void buildKeywordVertexIndex() {
//		Iterator<String> it = cache.keySet().iterator();
//		while (it.hasNext()) {
//			String keyword = it.next();
//			HashMap<Integer, SequentialIntArray> data = cache.get(keyword);
//			Iterator<Integer> it2 = data.keySet().iterator();
//			while (it2.hasNext()) {
//				Integer block = it2.next();
//				SequentialIntArray sia = data.get(block);
//				SubgraphMatrix graph = new SubgraphMatrix(blocks[block
//						.intValue() - 1]);
//				BlockKeyword bk = new BlockKeyword(block.intValue(), keyword);
//				GroupIteratorInSubgraph siis = new GroupIteratorInSubgraph(
//						graph, 0, 0, sia.getArray());
//				ArrayList<VertexDistance> list = new ArrayList<VertexDistance>();
//				while (siis.hasNext()) {
//					SPVertex spv = siis.next();
//					list.add(new VertexDistance(spv.getID(), (int) spv
//							.getDistance()));
//				}
//				try {
//					bidb.insertToKeywordVertex(bk, list);
//				} catch (DatabaseException ex) {
//					ex.printStackTrace();
//				}
//			}
//		}
//	}

	private void buildPortalVertexIndex() {
		for (int i = 0; i < blocks.length; i++) {
			Block block = blocks[i];
			SubgraphMatrix subgraph = new SubgraphMatrix(block);
			HashSet<Integer> portals = block.getPortals();
			Iterator<Integer> it = portals.iterator();
			while (it.hasNext()) {
				int portal = it.next().intValue();
//				if (block.getId() == 306 && portal == 49760) {
//					System.out.println();
//				}
				SimpleSubgraphVertex source = new SimpleSubgraphVertex(block
						.getId(), portal);
				DijkstraIteratorInSubgraph queue = new DijkstraIteratorInSubgraph(
						subgraph, source);
				ArrayList<PNEntry2> eia = new ArrayList<PNEntry2>();
				while (queue.hasNext()) {
					SPVertex spv = queue.next();
					if (spv.getDistance() == 0.0d) {
						continue;
					}
					PNEntry2 pne = new PNEntry2((short)spv.getDistance(), spv.getID(), spv.getPredecessor());
					eia.add(pne);
				}
				try {
					bidb.insertToPortalNode(source, eia);
				} catch (DatabaseException ex) {
					ex.printStackTrace();
				}
			}
			System.out.println("finished block " + i);
		}
	}

	private void buildVertexPortalIndex() {
		for (int i = 0; i < blocks.length; i++) {
			Block block = blocks[i];
			SubgraphMatrix subgraph = new SubgraphMatrix(block);
			HashSet<Integer> portals = block.getPortals();
			int[] vertexes = block.getVertexes();
			for (int j = 0; j < vertexes.length; j++) {
				SimpleSubgraphVertex source = new SimpleSubgraphVertex(block
						.getId(), vertexes[j]);
				if (portals.contains(new Integer(vertexes[j]))) {
					try {
						bidb.insertToVertexPortal(source, (short)0);
						continue;
					} catch (DatabaseException ex) {
						ex.printStackTrace();
					}
				}
				DijkstraIteratorInSubgraph queue = new DijkstraIteratorInSubgraph(
						subgraph, source);
				boolean mark = true;
				while (queue.hasNext()) {
					SPVertex spv = queue.next();
					int vertex = spv.getID();
					if (portals.contains(new Integer(vertex))) {
						try {
							bidb
									.insertToVertexPortal(source, (short)spv
											.getDistance());
							mark = false;
							break;
						} catch (DatabaseException ex) {
							ex.printStackTrace();
						}
					}
				}
				if (mark) {
					try {
						bidb.insertToVertexPortal(source, Short.MAX_VALUE);
					} catch (DatabaseException ex) {
						ex.printStackTrace();
					}
				}
			}
			System.out.println("finished block " + i);
		}
	}

	private HashSet<String> stopwordlist(String list) {
		HashSet<String> stopwordlist = new HashSet<String>();
		try {
			BufferedReader br = new BufferedReader(new FileReader(list));
			while (br.ready()) {
				stopwordlist.add(br.readLine().trim());
			}
			br.close();
		} catch (IOException ex) {
			ex.printStackTrace();
		}
		return stopwordlist;
	}
}
